#!/usr/bin/perl 
#
# rm_duplicates.pl
# Checks all the Message-IDs of email in subdirectories, and deletes duplicates
# from the subdir all_documents.
#
# invocation:
# rm_duplicates <full directory path>
#
# The MIT License
#
# Copyright (c) 2010 Andreas Fleig
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

use File::Find;

($root_dir) = @ARGV;

# gets all message ids from the subdirectory, stores them as keys in $ids
sub get_ids
{
    my ($root, $ids, $dir) = @_;
    my $dot = 0;

    sub add_id
    {
        my $fn = $root ."/". $File::Find::name;
        if (-f $fn) {
            open(IN, "$fn");
            my @content = <IN>;
            close(IN);
            foreach (@content) {
                if (/^Message-ID: <([^>]+)/) {
                    $ids->{$1} = 1;
                    break;
                }
            }
        }
        $dot = ($dot+1)%10;
        if ($dot == 0) {
            print ".";
            $|++;
        }
    }
    find(\&add_id, $dir);
}


sub rm_alldoc_duplicates
{
    my ($root, $ids, $dup) = @_;
    my $dot = 0;
    my $dir = $root . "/all_documents/";

    sub check_dup
    {
        my $fn = $File::Find::name;
        if (-f $fn) {
            open(IN, "$fn");
            my @content = <IN>;
            close(IN);
            my $mid = "";
            foreach (@content) {
                if (/^Message-ID: <([^>]+)/) {
                    $mid = $1;
                    break;
                }
            }
            if (exists($ids->{$mid})) {
                $dup++;
                unlink($fn);
            } else {
                $ids->{$mid} = 1;
            }

        }
        $dot = ($dot+1)%10;
        if ($dot == 0) {
            print "o";
            $|++;
        }
    }
    find(\&check_dup, $dir);
    return $dup;
}

# get subdirectories
#
opendir(D, $root_dir);
@contents = readdir(D);
closedir(D);

$all_documents = 0;
$subdirs = [ ];
foreach (@contents) {
    if (-d && $_ ne ".." && $_ ne ".") {
        if ($_ eq "all_documents") {
            $all_documents = 1;
        } else {
            push(@$subdirs, $_);
        }
    }
}

if ($all_documents) {

    # get message ids
    $ids = { };
    foreach (@$subdirs) {
        get_ids($root_dir, $ids, $_);
    }

    # get ids from all_documents, delete duplicates
    $dup_count = 0;
    $dup_count = rm_alldoc_duplicates($root_dir, $ids, $dup_count);

    print "\n";
    print "unique messages: ".keys(%$ids).", duplicates deleted: $dup_count\n";
}
